agora inbox for pgsql-hackers@postgresql.orghelp / color / mirror / Atom feed
[PATCH v10 1/3] Allow CREATE INDEX CONCURRENTLY on partitioned table 604+ messages / 2 participants [nested] [flat]
* [PATCH v10 1/3] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 604+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. XXX: does pgstat_progress_update_param() break other commands progress ? --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 141 ++++++++++++++++++------- src/test/regress/expected/indexing.out | 60 ++++++++++- src/test/regress/sql/indexing.sql | 18 +++- 4 files changed, 172 insertions(+), 56 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 749db2845e..ba4424d379 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -661,15 +661,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 996f1ed070..c1dd4c8362 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -93,6 +93,7 @@ static bool ReindexRelationConcurrently(Oid relationOid, int options); static void ReindexPartitions(Oid relid, int options, bool isTopLevel); static void ReindexMultipleInternal(List *relids, int options); +static void reindex_invalid_child_indexes(Oid indexRelationId); static void reindex_error_callback(void *args); static void update_relispartition(Oid relationId, bool newval); static bool CompareOpclassOptions(Datum *opts1, Datum *opts2, int natts); @@ -667,17 +668,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1111,6 +1101,11 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + { + /* If concurrent, initially build index partitions as "invalid" */ + flags |= INDEX_CREATE_INVALID; + } if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1163,6 +1158,14 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; Oid *part_oids = palloc(sizeof(Oid) * nparts); @@ -1170,12 +1173,15 @@ DefineIndex(Oid relationId, TupleDesc parentDesc; Oid *opfamOids; + // If concurrent, maybe this should be done after excluding indexes which already exist ? pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_TOTAL, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1218,10 +1224,12 @@ DefineIndex(Oid relationId, continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1292,10 +1300,14 @@ DefineIndex(Oid relationId, */ if (!found) { - IndexStmt *childStmt = copyObject(stmt); + IndexStmt *childStmt; bool found_whole_row; ListCell *lc; + oldcontext = MemoryContextSwitchTo(ind_context); + childStmt = copyObject(stmt); + MemoryContextSwitchTo(oldcontext); + /* * We can't use the same index name for the child index, * so clear idxname to let the recursive invocation choose @@ -1347,10 +1359,18 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PopActiveSnapshot(); + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1360,51 +1380,42 @@ DefineIndex(Oid relationId, * invalid, this is incorrect, so update our row to invalid too. */ if (invalidate_parent) - { - Relation pg_index = table_open(IndexRelationId, RowExclusiveLock); - HeapTuple tup, - newtup; - - tup = SearchSysCache1(INDEXRELID, - ObjectIdGetDatum(indexRelationId)); - if (!HeapTupleIsValid(tup)) - elog(ERROR, "cache lookup failed for index %u", - indexRelationId); - newtup = heap_copytuple(tup); - ((Form_pg_index) GETSTRUCT(newtup))->indisvalid = false; - CatalogTupleUpdate(pg_index, &tup->t_self, newtup); - ReleaseSysCache(tup); - table_close(pg_index, RowExclusiveLock); - heap_freetuple(newtup); - } - } + index_set_state_flags(indexRelationId, INDEX_DROP_CLEAR_VALID); + } else + table_close(rel, NoLock); /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) + { + if (concurrent) + reindex_invalid_child_indexes(indexRelationId); + pgstat_progress_end_command(); + } + return address; } - if (!concurrent) + table_close(rel, NoLock); + if (!concurrent || OidIsValid(parentIndexId)) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); + /* + * We're done if this is the top-level index, + * or the catalog-only phase of a partition built concurrently + */ - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } - /* save lockrelid and locktag for below, then close rel */ + /* save lockrelid and locktag for below */ heaprelid = rel->rd_lockInfo.lockRelId; SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); /* * For a concurrent build, it's important to make the catalog entries @@ -1586,6 +1597,56 @@ DefineIndex(Oid relationId, return address; } +/* Reindex invalid child indexes created earlier */ +static void +reindex_invalid_child_indexes(Oid indexRelationId) +{ + ListCell *lc; + int npart = 0; + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext; + List *childs = find_inheritance_children(indexRelationId, ShareLock); + List *partitions = NIL; + + PreventInTransactionBlock(true, "REINDEX INDEX"); + + foreach (lc, childs) + { + Oid partoid = lfirst_oid(lc); + + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + npart++); + + if (get_index_isvalid(partoid) || + !RELKIND_HAS_STORAGE(get_rel_relkind(partoid))) + continue; + + /* Save partition OID */ + oldcontext = MemoryContextSwitchTo(ind_context); + partitions = lappend_oid(partitions, partoid); + MemoryContextSwitchTo(oldcontext); + } + + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, REINDEXOPT_CONCURRENTLY); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + * This must be done only while holding a lock which precludes adding + * partitions. + * See also: validatePartitionedIndex(). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); +} /* * CheckMutability diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index 49b6f7c18f..2fb00a2fa7 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,63 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2_ccnew" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) INVALID + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) INVALID +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + "idxpart2_a_idx2_ccnew" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/20180124162006.pmapfiznhgngwtjf@alvherre.pgsql diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index fc1479dca6..610c12de9e 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,22 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --i3lJ51RuaGWuFYNw Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v10-0002-Add-SKIPVALID-flag-for-more-integration.patch" ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 604+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 604+ messages in thread
end of thread, other threads:[~2026-07-15 08:37 UTC | newest] Thread overview: 604+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2020-06-06 22:42 [PATCH v10 1/3] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <pryzbyj@telsasoft.com> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <ah@cybertec.at>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox